import time
import logging

import requests
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities


# LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
# logging.basicConfig(filename='scrapy.log', level=logging.DEBUG,
#                     format=LOG_FORMAT)

def douyu_main():
    time.sleep(3)
    with webdriver.Remote("http://127.0.0.1:4444/wd/hub",
                          DesiredCapabilities.CHROME) as driver:
        url = 'https://www.douyu.com/directory/all'
        web_url = "http://127.0.0.1:8000/douyu/data/"

        driver.get(url)
        driver.implicitly_wait(10)
        time.sleep(3)

        title_xpath = '//*[@id="listAll"]/section[2]/div[2]/ul/li[{}]//h3'
        url_path = '//*[@id="listAll"]/section[2]/div[2]/ul/li[{}]//a/@href'
        cate_xpath = '//*[@id="listAll"]/section[2]/div[2]/ul/li[{}]//span[@class="DyListCover-zone"]'
        auth_xpath = '//*[@id="listAll"]/section[2]/div[2]/ul/li[{}]//div[@class="DyListCover-userName"]'
        hot_xpath = '//*[@id="listAll"]/section[2]/div[2]/ul/li[{}]//span[@class="DyListCover-hot"]'
        next_page_xpath = '//span[text()="下一页"]'

        page = 1
        while True:
            result = []
            time.sleep(3)
            for i in range(1, 120 + 1):
                title = driver.find_element_by_xpath(
                    title_xpath.format(i)).text.strip()
                cate = driver.find_element_by_xpath(
                    cate_xpath.format(i)).text.strip()
                auth = driver.find_element_by_xpath(
                    auth_xpath.format(i)).text.strip()
                hot = driver.find_element_by_xpath(
                    hot_xpath.format(i)).text.strip()

                li = {'title': title, 'cate': cate, 'auth': auth, "hot": hot}
                # result.update(li)
                result.append(li)
                print("page: {}, index:{}, value:{}".format(page, i, li))
                # logging.info("page: {}, index:{}, value:{}".format(page, i, li))
            print("发起post请求 发送第{}页数据".format(page)) # TODO
            requests.post(web_url, json=result)
            # 翻下一页
            try:
                next_page = driver.find_element_by_xpath(next_page_xpath)
                next_page.click()
                page += 1
            except Exception as e:
                print("翻页失败{}".format(e))
                # logging.debug("翻页失败   {}".format(e))
                break


if __name__ == '__main__':
    douyu_main()
